#!/usr/bin/env bash
set -euo pipefail

DATA_DIR="$(dirname "$0")/../data"

for csv in "$DATA_DIR"/*.csv; do
  parquet="${csv%.csv}.parquet"
  if [ ! -f "$parquet" ]; then
    echo "WARNING: parquet file not found for $csv -> $parquet"
    continue
  fi
  abs_csv=$(python3 -c "import pathlib,sys;print(pathlib.Path(sys.argv[1]).resolve())" "$csv")
  abs_parquet=$(python3 -c "import pathlib,sys;print(pathlib.Path(sys.argv[1]).resolve())" "$parquet")

  echo
  echo "== Checking $abs_csv vs $abs_parquet =="

  # row counts
  duckdb -c "SELECT 'parquet_rows' AS what, count(*) AS n FROM read_parquet('$abs_parquet');"
  duckdb -c "SELECT 'csv_rows' AS what, count(*) AS n FROM read_csv_auto('$abs_csv');"

  # set up tables and compare sets (EXCEPT)
  duckdb -c "CREATE TABLE p AS SELECT * FROM read_parquet('$abs_parquet'); CREATE TABLE c AS SELECT * FROM read_csv_auto('$abs_csv'); SELECT 'p_minus_c' AS diff, count(*) FROM (SELECT * FROM p EXCEPT SELECT * FROM c); SELECT 'c_minus_p' AS diff, count(*) FROM (SELECT * FROM c EXCEPT SELECT * FROM p); DROP TABLE p; DROP TABLE c;"
done

echo
echo "All checks finished."
